# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import csv
import hashlib
import json
import os

# 查重
from redis import StrictRedis
from scrapy.exceptions import DropItem


class CheckExist:
    def open_spider(self, spider):
        # 链接Redis
        self.redis_client = StrictRedis(host='localhost', port=6379, db=0)

    def close_spider(self, spider):
        # 关闭Redis的链接
        self.redis_client.close()

    def process_item(self, item, spider):
        msg_type = item.get("type")
        if msg_type == "info":
            print("----Redis检查管道---")
            waitting_hash_str = json.dumps(item)
            md5 = hashlib.md5()
            md5.update(waitting_hash_str.encode())
            hash_val = md5.hexdigest()
            # 1. 判断Redis中是否存在
            exist_flag = self.redis_client.get(hash_val)

            if exist_flag:
                # 2. 如果有则抛出异常
                raise DropItem("----此数据丢弃，因为Redis中查询到了----")
            else:
                # 3. 如果没有则存储到Redis且返回item
                self.redis_client.set(hash_val, waitting_hash_str)
            return item
        else:
            return item


class ThepaperPipeline:
    def __init__(self):
        self.download_path = os.getcwd() + '\images\\'
        self.download_path_csv = os.getcwd()

    def open_spider(self, spider):
        # 链接Redis
        self.f = open('澎湃新闻.csv', 'a', encoding='utf-8')
        self.f_csv = csv.DictWriter(self.f, ['title', 'img_src', 'link_url'])
        self.f_csv.writeheader()

    def close_spider(self, spider):
        # 关闭Redis的链接
        self.f.close()

    def process_item(self, item, spider):
        print('----acacac-----')
        # 当前文件夹下的download文件夹
        if not os.path.exists(self.download_path):  # 判断文件夹或文件
            os.makedirs(self.download_path)

        msg_type = item.get("type")
        if msg_type == "info":
            item.pop("type")  # 删除type 这个key-value
            self.f_csv.writerows([item])
            print("保存信息到CSV....ok")
        elif msg_type == "img":
            # 存储图片
            img_name = item.get("img_name")
            img_bytes = item.get("img_bytes")
            with open(self.download_path + img_name, "wb") as f:
                f.write(img_bytes)
                print("保存图片%s完成....ok" % img_name)
